package timeline;

import java.io.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;

public class Util {
	
	private static Util instance = null;
	private static String cmd = "wget";
	private static String timeoutStr = "--timeout 300";
	private static String useragent_filename  = "../config/useragentstring";
	private String recurive_para = "-p -r -l1";
	private static final  String output_root = "../output/webpage";
	private ArrayList<String[]> userAgentList;
	private static boolean createIndexFile;
	public static Util getInstance(boolean inputcreateIndexFile) {
	      if(instance == null) {
	    	 createIndexFile = inputcreateIndexFile;
	         instance = new Util();
	      }
	      return instance;
	}

	      
	private long serial;
	PrintStream infoFile = null;
	protected Util()
	{
		if (createIndexFile)
		{
			//read serial number from index file;
			serial = getSerial();
			
			try{
				boolean exists = (new File(output_root)).exists();
				if (!exists) 
		        {
		        	boolean success = (new File(output_root)).mkdir();
		            if (!success) 
		            {
		              System.out.println("Create Directory: " + output_root + "Failed. Exit");
		              return;
		            }
		         }
				infoFile = new PrintStream (new FileOutputStream(output_root+"/index.result", true), true);
				infoFile.println("GetTime\tCreateTime\tUser\tURL\tSerialNum");
			}
		catch(Exception exception) {exception.printStackTrace(); System.exit(0);}
		}
		try{
			userAgentList = new ArrayList<String[]>(); 			
			readUserAgentFile();
		}
		catch(Exception exception) {exception.printStackTrace(); System.exit(0);}
		
	}
	
	private long getSerial()
	{
		String lastValidLine = null;
		String lastSecondValidLine = null;
		try{
			
			String indexFile = output_root+"/index.result";
		    File f = new File(indexFile);
		    if (!f.exists()) return 0;
		    FileReader fr = new FileReader(indexFile);
		    BufferedReader br = new BufferedReader(fr);
			   String line;
			   Pattern p = Pattern.compile("[0-9]+.*");
			   
			   System.out.println("Scan index file.........");

			   
		       while(true)
		       {
		    	   line = br.readLine();
		    	   //System.out.println("Line: "+line);
		    	   if (line == null) break;
		           if (line.length()==0) continue;
		           Matcher m = p.matcher(line);
				   if(! m.matches())
				   {
					   System.out.println("Not vaild line");
					   
				   }
				   else
				   {
					   lastSecondValidLine =lastValidLine; 
					   lastValidLine = line;
				   }
				   
		       }
		       br.close();
		       fr.close();
		       System.out.println("Last Valid Line: "+lastValidLine);		       
		       String lastserial_str = lastValidLine.substring(lastValidLine.length()-10, lastValidLine.length());
		       long lastserial = Long.parseLong(lastserial_str);
		       System.out.println("last serial: "+lastserial);
		       
		       return lastserial+1;
			   }
		       catch( Exception e ) 
		  	      {
		  	         System.out.println("Last Line is not valid! Try Last second line");
		  	      }
		       try
		       {
		       System.out.println("Last Second Valid Line: "+lastSecondValidLine);		       
		       String lastserial_str = lastSecondValidLine.substring(lastSecondValidLine.length()-10, lastSecondValidLine.length());
		       long lastserial = Long.parseLong(lastserial_str);
		       System.out.println("last second serial: "+lastserial);
		       
		       return lastserial+1;
			   }
		       catch( Exception e ) 
		  	      {
		  	         System.out.println("Last second Line is not valid! Return 2000000000");
		  	      }
		  	         
		  	         
		  	         //return;
		  	         return 2000000000;
		  	       
	}
	protected void finalize()
	{
		infoFile.close();
	}
	
	
	
	//Directory structure: ../output/webpage/<DateTime>/<url'serial number>/<query time(0,24,28>/<user agent string>/ 
	public void wget(Date createTime,String info, String url,long subdir_count, int repeat_time)
	{
		//System.out.println(info+" "+url+" "+subdir_count+" "+repeat_time);
		//System.out.println("Read Url: "+url);
		Calendar cal = Calendar.getInstance();
		cal.clear();
		cal.setTime(createTime);
        SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd_HHmmssZ");
        String currenttime = sdf.format(cal.getTime());

        SimpleDateFormat sdf2 = new SimpleDateFormat("yyyyMMdd");
        String currentdate = sdf2.format(cal.getTime());
        
        ///create dir by date
        String dateleveldir = output_root +"/"+currentdate;
        boolean exists = (new File(dateleveldir)).exists();
        if (!exists) 
        {
        	boolean success = (new File(dateleveldir)).mkdir();
            if (!success) 
            {
              System.out.println("Create Directory: " + dateleveldir + "Failed. Exit");
              return;
            }
         }
        
        String current_serial;
        
        if (repeat_time ==0)
        {
        	DecimalFormat f = new DecimalFormat( "0000000000");
        	current_serial = f.format(serial);
        	
        	//protect serial number to be unique 
        	synchronized(this)
        	{
        		serial++;
        	}
            infoFile.println (currenttime+"\t"+info+"\t"+current_serial);
            infoFile.flush();
        	
        }
        else {
        	current_serial = info;
        	//System.out.println("current_serial: "+current_serial);
        }
        
        //create dir for this page
    	String pageleveldir = dateleveldir+"/"+current_serial;
    	exists = (new File(pageleveldir)).exists();
        if (!exists) 
        {
        	boolean success = (new File(pageleveldir)).mkdir();
        	if (!success) 
        	{
        	      System.out.println("Error: Directory: " + pageleveldir + "can not be created");
        	}
        	
        }
    	
    	//create dir for this time  
        String couteleveldir = pageleveldir +"/"+subdir_count;
        exists = (new File(couteleveldir)).exists();
        if (!exists) 
        {
        	boolean success = (new File(couteleveldir)).mkdir();
            if (!success) 
            {
              System.out.println("Create Directory: " + couteleveldir + "Failed. Exit");
              return;
            }
        }
        
    	
    	try{
     	    //wget currentpage
    		Iterator<String[]> iter = userAgentList.iterator();
            
            while (iter.hasNext())
            {
            	String[] agentPair = iter.next();
    			String agentleveldir = couteleveldir +"/"+agentPair[0];
    			//System.out.println("current user agent:"+agentPair[0]);
    			
    			exists = (new File(agentleveldir)).exists();
    	        if (!exists) 
    	        {
    	        	boolean success = (new File(agentleveldir)).mkdir();
        	        if (!success) 
        	        {
        	           System.out.println("Create Directory: " + agentleveldir + "Failed. Exit");
        	           return;
        	         }
    	        	
    	        }    	    	String logfile = agentleveldir+"/"+"wget_currentpage.log";
    			execWget (url,agentPair[1],agentleveldir,logfile, false);
    	    	
    			//Due to space problem, remove recusive wget
    	    	/*String recuriveUrl = getRecusiveUrl (logfile);
    	    	if (recuriveUrl != null)
    	    	{
    	    		logfile = agentleveldir+"/"+"wget_recurive.log";
    	    		execWget (recuriveUrl,agentPair[1],agentleveldir,logfile, true);	
    	    	}*/
    		}
    	}
    	catch(Exception exception) {exception.printStackTrace();}
	}
	
	private void readUserAgentFile() throws Exception
	{
		FileReader fr = new FileReader(useragent_filename);
		BufferedReader br = new BufferedReader(fr);
		String line;
        
       while(true)
       {
    	   line = br.readLine();
    	   if (line == null) break;
           if (line.length()==0) continue;
           
           String agentPair[] = new String[2];
           agentPair[0] = line.substring(0,line.indexOf(" "));
           agentPair[1] = line.substring(line.indexOf(" ")+1);
           userAgentList.add (agentPair);
           System.out.println ("User agent:"+agentPair[0]+" agent string: "+agentPair[1].replace('#', ' '));
        }

	}
	
	private String getRecusiveUrl(String logfile) throws Exception
	{
		FileReader fr = new FileReader(logfile);
        BufferedReader br = new BufferedReader(fr);
        
        String line;
        String finalURL = null;
        
       while(true)
       {
    	   line = br.readLine();
    	   if (line == null) break;
           if (line.length()==0) continue;
           ArrayList<String> urlList = getUrl(line);
           if (!urlList.isEmpty())
           {
        	   finalURL = urlList.get(urlList.size()-1);
           }
        }
       if (finalURL == null)
       {
    	   System.out.println ("Error: no URL in wget.log file! file:"+logfile);
    	   return null;
       }
       int idx = finalURL.lastIndexOf('/');
       if (idx == finalURL.length()-1) idx=finalURL.lastIndexOf('/',finalURL.length()-2); 
       //String upperURL = finalURL.substring(0,idx);
       //System.out.println("URL:" +finalURL);
       //System.out.println("upper URL:" +upperURL);
       return finalURL;
		
	}
	
	private void execWget( String url,String useragent, String outputdir,String logfilename, boolean recursive) throws Exception
	{
    
	    String s = null;
    	Process p;
    	
       	String cmdLine = cmd+" "+timeoutStr+" --user-agent="+"\\\""+useragent+"\\\""+" -P " +outputdir+" -o " +logfilename+" ";
       	if (recursive ){
       		cmdLine = cmdLine+recurive_para+" ";
       	}
       	
       	cmdLine = cmdLine+url;
       	//cmdLine="/bin/sh -c \""+cmdLine+"\"";
      
    	//System.out.println (cmdLine.replace('#', ' '));
    	System.out.println ("Wget. DIR: "+outputdir+ " URL: "+url);

    	//p = Runtime.getRuntime().exec(cmdLine);
    	
    	String[] cmdA;
       	if (recursive ){
       		cmdA = new String[12];
       	}
       	else {
       		cmdA = new String[9];
       	}
        StringTokenizer st = new StringTokenizer(cmdLine);
        int i=0;
        while (st.hasMoreTokens()) {
            cmdA[i] = st.nextToken();
            if (i==3)
            	cmdA[i] = cmdA[i].replace('#', ' ');
            i++;
        }
        
//    	cmdA[0] = cmd;
//    	cmdA[1] = "--timeout";37.3%us,  3.9%sy,
//    	cmdA[2] = "300";
//    	cmdA[3] = "--user-agent="+"\""+useragent+"\"";
//    	cmdA[4] = "-P";
//    	cmdA[5] = outputdir;
//    	cmdA[6] = "-o";
//    	cmdA[7] = logfilename;
//       	if (recursive ){
//       		cmdA[8] = "-p";
//       		cmdA[9] = "-r";
//       		cmdA[10] = "-l1";
//       		cmdA[11] = url;
//       	}
//       	else{
//       		cmdA[8] = url;
//       		cmdA[9] = "";
//       		cmdA[10] = "";
//       		cmdA[11] = "";
//       	}
       	
    	p = Runtime.getRuntime().exec(cmdA);
    	BufferedReader stdInput = new BufferedReader(new 
                InputStreamReader(p.getInputStream()));

           BufferedReader stdError = new BufferedReader(new 
                InputStreamReader(p.getErrorStream()));

           // read the output from the command
           while ((s = stdInput.readLine()) != null) {
               System.out.println(s);
           }
           
           // read any errors from the attempted command

           while ((s = stdError.readLine()) != null) {
               System.out.println(s);
           }
           p.waitFor();
           
	}
	
	public ArrayList<String> getUrl( String line)
	{

		ArrayList<String> urlList = new ArrayList<String>();
		
       	line = line.replaceAll("[^\\x00-\\x7F]+"," ");
    	line = line.replaceAll("[\\t]+"," ");
        //System.out.println(line);
        
    	int header = 0;
        while (header<line.length())
        {
        	int sign_index =  line.indexOf("://", header);
        	if (sign_index <0 ) break;
        	int begin_index = line.lastIndexOf(' ', sign_index);
            int end_index = line.indexOf(' ', sign_index);
            if (end_index <0 ) end_index = line.length();
            header = end_index;
          
            String url = line.substring(begin_index+1, end_index);
            String orig_url = url;
            
            while (!url.matches("[\\w]+://+.*"))
            {
            	url = url.replaceFirst("^[a-z]*[^a-z]+","");
            	//System.out.println(" URL: "+url);
            }
            
            url = url.replaceAll("[^\\w]+$","");
            
            if (!url.matches("[\\w]+://[\\w]+.*"))
            {
            	System.out.println("Invalid URL: "+url);
            	System.out.println("Invalid orig URL: "+orig_url);
            	continue;
            }
            urlList.add (url);
            
	}

     return urlList;    
    
}
	
public String getDomain ( String url)
{
	String[] fields = url.split("/+");
    return fields[1];

}

}
    

